Demonstration of using python generators as coroutines to implement cooperative multi-tasking

coroutine_generator_demo.py

"""
Demonstration of using python generators as coroutines to implement
cooperative multi-tasking. Catting multiple files in chunks to stdout.

There is room for much more generationization of operators, for example
implementing the yielding value as a Future.

Pass in a list of file names.
"""
import sys
import os
import select
MAX_BUF = 20

def cat(fname):
    fd = os.open(fname, os.O_RDONLY | os.O_NONBLOCK)
    while True:
        line = yield fd
        if not line:
            break
        print "%s:%r" % (fname, line)

def ioloop():
    # List of generators
    running_cats = map(cat, sys.argv[1:])

    # map file descriptors to list of cats waiting on data
    hungry_cats = {}

    # food for the cats to eat. map generator worker to returned data
    food = {}

    while running_cats or hungry_cats:
        feed_cats(running_cats, food, hungry_cats)
        food = food_for(hungry_cats)
        running_cats.extend(food.keys())

def feed_cats(running, food, hungry):
    # Let them run
    while running:
        c = running.pop()
        try:
            fd = c.send(food.pop(c, None))
        except StopIteration:
            #  Job Done
            continue
        hungry[fd] = c

def food_for(hungry_cats):
    # See who is ready to wake up
    food = {}
    rlist, _, _ = select.select(hungry_cats.keys(), [], [], 0)
    for fd in rlist:
        data = os.read(fd, MAX_BUF)
        food[hungry_cats.pop(fd)] = data
    return food

ioloop()