Skip to content

Instantly share code, notes, and snippets.

@fission6
Created December 8, 2011 04:33
Show Gist options
  • Save fission6/1446124 to your computer and use it in GitHub Desktop.
Save fission6/1446124 to your computer and use it in GitHub Desktop.
gevent concurrent url request snippet
import gevent
from gevent import monkey; monkey.patch_all()
from gevent import pool
import random
from time import sleep, time
import urllib2
POOL_SIZE = 5
def yield_keyword():
keywords_list = ['http://www.yahoo.com/'] * 5
for entry in keywords_list:
yield entry
def worker(entry):
print "Worker - Fetching for {0}".format(entry)
content = urllib2.urlopen(entry).read()
print "Worker - Completed for {0}".format(content)
print "**Spinning Down a Worker**"
worker_pool = pool.Pool(POOL_SIZE)
def main():
start_time = time()
for entry in yield_keyword():
worker_pool.spawn(worker, entry)
print "**Spinning Up a Worker**"
# block until all tasks are done
worker_pool.join()
#timestamp
run_time = (time() - start_time) / 60
print "Total Runtime:", run_time
if __name__ == '__main__':
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment