Skip to content

Instantly share code, notes, and snippets.

@littlecodersh
Created October 29, 2016 14:52
Show Gist options
  • Save littlecodersh/495265d0fe7e65bcf464e03d9329bbdd to your computer and use it in GitHub Desktop.
Save littlecodersh/495265d0fe7e65bcf464e03d9329bbdd to your computer and use it in GitHub Desktop.
from tornado import httpclient, gen, ioloop, locks
def fetch(urlList, minRequire):
l = locks.Semaphore(len(urlList))
resultList = []
@gen.coroutine
def _fetch():
@gen.coroutine
def __fetch(url):
r = yield httpclient.AsyncHTTPClient().fetch(url)
l.release()
resultList.append(r.body)
print('Fetched: %s' % url)
for i in range(len(urlList)): l.acquire()
for url in urlList: __fetch(url)
for i in range(minRequire): yield l.acquire()
ioloop.IOLoop.current().run_sync(_fetch)
return resultList
if __name__ == '__main__':
urlList = ['http://127.0.0.1:5000/%s' % i for i in range(5)]
r = fetch(urlList, 3)
print(r)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment