Network co-routines with gevent

gistfile1.py

#### GEVENT
import gevent
import gevent.monkey
import gevent.pool

def map_gevent(func, it, pool_size = None):
    pool_size = pool_size if pool_size is not None else 10

    pool = gevent.pool.Pool(pool_size)
    result = []

    def worker(index, i):
        ret = func(i)
        result.append((index, ret))

    for index, i in enumerate(it):
        pool.spawn(worker, index, i)

    pool.join()
    result.sort(key = lambda i: i[0])
    result = [i[1] for i in result]

    return result

#### PROFILING
import time
import contextlib

@contextlib.contextmanager
def profile():
    before = time.time()
    yield
    after = time.time()
    laps = after - before

    print "Profiling: %fs" % (laps, )

#### USER CODE
import urllib2
import sys

def urllib_read(url):
    print "Opening %r..." % (url, )
    return urllib2.urlopen(url).read()

def main():
    pool_size = int(sys.argv[1]) if len(sys.argv) > 1 else 1

    gevent.monkey.patch_all()

    urls = [
        "http://homengo.com/",
        "http://google.com/",
        "http://perdu.com/",
        "http://allmyapps.com/",
    ]

    urls = urls * 10

    print "%d requests..." % (len(urls), )
    print "Pool size:", pool_size

    with profile():
        results = map_gevent(urllib_read, urls, pool_size)

if __name__ == "__main__":
    main()