Skip to content

Instantly share code, notes, and snippets.

@rafaelnovello
Created February 23, 2018 12:41
Show Gist options
  • Save rafaelnovello/c23f3b86ae13ac8b92bc638a459b8d98 to your computer and use it in GitHub Desktop.
Save rafaelnovello/c23f3b86ae13ac8b92bc638a459b8d98 to your computer and use it in GitHub Desktop.
from datetime import datetime
links = [
'https://g1.globo.com/rj/rio-de-janeiro/noticia/pf-cumpre-mandados-em-mais-um-desdobramento-da-lava-jato-no-rio.ghtml',
'https://g1.globo.com/sp/sao-paulo/noticia/ataque-a-tiros-mata-1-e-deixa-2-feridas-em-frente-a-hotel-em-sp-veja-video.ghtml',
'https://globoesporte.globo.com/olimpiadas-de-inverno/noticia/nervosa-isadora-williams-sofre-queda-na-final-da-patinacao-e-chora-muito-triste.ghtml',
'https://g1.globo.com/agenda-do-dia/noticia/sexta-feira-23-de-fevereiro.ghtml',
'https://g1.globo.com/rj/rio-de-janeiro/noticia/general-richard-nunez-sera-o-secretario-de-seguranca-do-rj.ghtml',
'https://g1.globo.com/rj/rio-de-janeiro/noticia/comandante-de-instituicao-de-ensino-da-pm-do-rj-e-exonerado-policiais-denunciam-abusos.ghtml',
'https://g1.globo.com/politica/noticia/sera-entre-ruim-e-tragico-se-o-stf-reverter-a-prisao-apos-2-instancia-diz-barroso.ghtml',
'https://g1.globo.com/rj/rio-de-janeiro/noticia/estudo-indica-que-maioria-dos-acusados-por-trafico-no-rj-nao-tem-antecedentes-nem-foi-investigada.ghtml',
'https://g1.globo.com/sp/sao-paulo/noticia/video-mostra-desespero-de-hospedes-de-hotel-durante-ataque-a-suspeito-de-integrar-faccao-em-sp.ghtml',
'https://g1.globo.com/ciencia-e-saude/blog/cassio-barbosa/post/2018/02/23/o-que-se-passa-com-o-universo.ghtml'
]
def do_get_by_threads(links):
import requests
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=3) as ex:
for resp in ex.map(requests.get, links):
print(resp.status_code)
start = datetime.now()
do_get_by_threads(links)
print(datetime.now() - start)
async def fetch(url):
import aiohttp
import async_timeout
session = aiohttp.ClientSession()
async with async_timeout.timeout(10):
async with session.get(url, timeout=10) as response:
return response
def do_get_by_asyncio(links):
import asyncio
asyncio.set_event_loop(asyncio.new_event_loop())
tasks = [asyncio.ensure_future(fetch(url)) for url in links]
loop = asyncio.get_event_loop()
responses = loop.run_until_complete(asyncio.gather(*tasks, return_exceptions=True))
for r in responses:
print(r.status)
start = datetime.now()
do_get_by_asyncio(links)
print(datetime.now() - start)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment