Skip to content

Instantly share code, notes, and snippets.

@morrisalp
Last active September 18, 2023 12:19
Show Gist options
  • Save morrisalp/65dfdadd756ec1e9fa7566e539eda87a to your computer and use it in GitHub Desktop.
Save morrisalp/65dfdadd756ec1e9fa7566e539eda87a to your computer and use it in GitHub Desktop.
send async HTTP requests using grequests with tqdm progress bar
from tqdm import tqdm
import requests, grequests
class ProgressSession():
def __init__(self, urls):
self.pbar = tqdm(total = len(urls), desc = 'Making async requests')
self.urls = urls
def update(self, r, *args, **kwargs):
if not r.is_redirect:
self.pbar.update()
def __enter__(self):
sess = requests.Session()
sess.hooks['response'].append(self.update)
return sess
def __exit__(self, *args):
self.pbar.close()
def get_urls_async(urls):
with ProgressSession(urls) as sess:
rs = (grequests.get(url, session = sess, timeout = 5) for url in urls)
return grequests.map(rs)
@GeorgeKontsevik
Copy link

fire

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment