Skip to content

Embed URL

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Simple script for faster fetching with urllib2 with exception handling
import threading, urllib2
import Queue
def read_url(url, queue):
try:
data = urllib2.urlopen(url).read()
except urllib2.HTTPError, e:
checksLogger.error('HTTPError = ' + str(e.code))
except urllib2.URLError, e:
checksLogger.error('URLError = ' + str(e.reason))
except httplib.HTTPException, e:
checksLogger.error('HTTPException')
except Exception:
import traceback
checksLogger.error('generic exception: ' + traceback.format_exc())
print('Fetched %s from %s' % (len(data), url))
queue.put(data)
def fetch_parallel(list_of_urls):
result = Queue.Queue()
threads = [threading.Thread(target=read_url, args = (url,result)) for url in list_of_urls]
for t in threads:
t.start()
for t in threads:
t.join()
return result
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Something went wrong with that request. Please try again.