Skip to content

Instantly share code, notes, and snippets.

@tsujeeth
Last active August 29, 2015 14:15
Show Gist options
  • Save tsujeeth/88243610a703f61b3428 to your computer and use it in GitHub Desktop.
Save tsujeeth/88243610a703f61b3428 to your computer and use it in GitHub Desktop.
Async HTTP Client, using Tornado, to fetch a list of URLs
# The list of URLs is provide in an input file.
# (assuming one URL per line)
from tornado import httpclient
from tornado import gen
from tornado.ioloop import IOLoop
class Fetcher:
def __init__(self):
self.client = httpclient.AsyncHTTPClient()
@gen.coroutine
def getPage(self, url):
response = yield self.client.fetch(url)
raise gen.Return(response)
def printPage(future_page):
print future_page.result()
def makeRequest():
fetcher = Fetcher()
f = open('urls.txt', 'r')
for url in f:
IOLoop.current().add_future(fetcher.getPage(url.rstrip()), printPage)
f.close()
if __name__ == "__main__":
IOLoop.instance().add_callback(makeRequest)
IOLoop.instance().start()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment