Created
September 4, 2017 05:06
-
-
Save gnilchee/29688b592fc746a55cea92633da1de29 to your computer and use it in GitHub Desktop.
python3 concurrency example with blocking requests
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import requests | |
my_urls = [ | |
'https://google.com/', | |
'https://youtube.com/', | |
'https://facebook.com/', | |
'https://wikipedia.org/', | |
'https://yahoo.com/', | |
'https://reddit.com/', | |
'https://amazon.com/', | |
'https://github.com/', | |
'https://twitter.com', | |
'https://news.google.com/', | |
'https://instagram.com/', | |
'https://httpbin.org', | |
'https://linkedin.com/', | |
'https://netflix.com/', | |
'https://imgur.com/', | |
'https://ebay.com/', | |
'https://pornhub.com/', | |
'https://wordpress.com/', | |
'https://bing.com/', | |
'https://microsoft.com/' | |
] | |
def get_urls(url): | |
r = requests.get(url) | |
result = "{url} {status}".format(url=r.url, status=r.status_code) | |
return result | |
for url in my_urls: | |
print(get_urls(url)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
import asyncio | |
import concurrent.futures | |
import requests | |
my_urls = [ | |
'https://google.com/', | |
'https://youtube.com/', | |
'https://facebook.com/', | |
'https://wikipedia.org/', | |
'https://yahoo.com/', | |
'https://reddit.com/', | |
'https://amazon.com/', | |
'https://github.com/', | |
'https://twitter.com', | |
'https://news.google.com/', | |
'https://instagram.com/', | |
'https://httpbin.org', | |
'https://linkedin.com/', | |
'https://netflix.com/', | |
'https://imgur.com/', | |
'https://ebay.com/', | |
'https://pornhub.com/', | |
'https://wordpress.com/', | |
'https://bing.com/', | |
'https://microsoft.com/' | |
] | |
async def main(): | |
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: | |
loop = asyncio.get_event_loop() | |
futures = [ | |
loop.run_in_executor( | |
executor, | |
requests.get, | |
url | |
) | |
for url in my_urls | |
] | |
for response in await asyncio.gather(*futures): | |
print("{url} {status}".format(url=response.url, status=response.status_code)) | |
loop = asyncio.get_event_loop() | |
loop.run_until_complete(main()) |
Author
gnilchee
commented
Sep 4, 2017
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment