Skip to content

Instantly share code, notes, and snippets.

@mhrivnak
Last active December 15, 2015 06:39
Show Gist options
  • Save mhrivnak/5217649 to your computer and use it in GitHub Desktop.
Save mhrivnak/5217649 to your computer and use it in GitHub Desktop.
#!/usr/bin/env python
import eventlet
from eventlet.green import urllib2
def fetch(url):
body = urllib2.urlopen(url).read()
pool = eventlet.GreenPool(15)
with open('files.txt') as f:
urls = f.readlines()
for result in pool.imap(fetch, urls):
pass
#!/usr/bin/env python
import eventlet
from eventlet.green import urllib2
def fetch(url):
response = urllib2.urlopen(url)
name = url.rsplit('/', 1)[1]
with open(name, 'w') as f:
f.write(response.read())
pool = eventlet.GreenPool(7)
with open('files.txt') as f:
urls = f.readlines()
for result in pool.imap(fetch, urls):
pass
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment