Queuing fabric tasks and stream reading output using celery & redis - A little bit of evil with tempfiles and stdout redirection.

tasks.py

# RUN ME, I am the worker!
# $ pip install fabric celery-with-redis
# $ celery -A tasks worker -E --loglevel=debug
from celery import Celery
from time import sleep
from fabric.api import env, run, execute
import sys

celery = Celery('tasks', broker='redis://', backend='redis://')

@celery.task(track_started=True)
def run_commands(output_filename):
    output = open(output_filename, 'w')
    env.host_string = 'localhost'
    sys.stdout = output
    sleep(1)
    run("uname;uname;uname;uname;uname")
    sleep(2)
    run("uname -a")
    run("sleep 2")
    run("tty")
    sleep(2)
    run("echo DONE!")
    sys.stdout = sys.__stdout__

test.py

#!/usr/bin/env python
# RUN ME, I queue the task then read the output
# python ./test.py
from tasks import run_commands
from time import sleep
from tempfile import NamedTemporaryFile

output = NamedTemporaryFile()
print "created outputfile: %s" % output.name

# Queue up a new task.
task = run_commands.delay(output_filename=output.name)

print "print fabfile output as it happens"
# Keep looping until the task has finished.
while True:
    new_printable_output = output.read()
    if new_printable_output:
        print new_printable_output
    if task.state not in ['PENDING', 'STARTED']:
       break
    sleep(.5)
print "*** Fabric task has finished ***"

print "-"*40
print "- printing all out the output from the fabric task -"
print "-"*40
output.seek(0)
print output.read()