ondemand_scheduler.py

from IPython.parallel import Client
from IPython.parallel import interactive
import time
import random

@interactive
def work(i):
    import time
    import random
    sleeptime = 1 + random.random() * 2
    time.sleep(sleeptime)
    return i, sleeptime

client = Client()

dv = client.direct_view()

promises = []
results = []

submit_idx = 0
t0 = time.time()

available = set(client.ids)

while True:
    for eid in list(available):
        print 'generate some work for', eid
        promises.append(
            client[eid].apply_async(work, submit_idx)
        )
        available.remove(eid)
        submit_idx += 1
    
    new_promises = []
    for p in promises:
        if p.ready():
            results.append(p.get())
            available.add(p.engine_id)
        else:
            new_promises.append(p)
    promises = new_promises
    print 'n engines', len(client),
    print 'total time: ', (time.time() - t0),
    if results:
        print 'time spent in workers', sum(zip(*results)[1])
    else:
        print
    time.sleep(.5)