Skip to content

Instantly share code, notes, and snippets.

@SureshKL
Created January 23, 2019 11:55
Show Gist options
  • Save SureshKL/86be23f2de549601345259ab1bdc1fd3 to your computer and use it in GitHub Desktop.
Save SureshKL/86be23f2de549601345259ab1bdc1fd3 to your computer and use it in GitHub Desktop.
Python - Single vs Multithreaded download using threading and gevent
import logging
import threading
import gevent
import gevent.monkey
import requests
gevent.monkey.patch_socket()
logging.basicConfig(
format='%(asctime)s - %(levelname)-8s - [%(threadName)-10s-%(thread)5d] - %(message)s',
level=logging.DEBUG)
logger = logging.getLogger(__name__)
urls = ['https://www.python.org/',
'https://realpython.com/',
'http://www.tutorialspoint.com/python/']
proxies = {}
def download_via_thread(url):
return threading.Thread(target=requests.get,
kwargs={'url': url, 'proxies': proxies})
def download_via_gevent(url):
logger.info(f'Download {url} via gevent')
requests.get(url, proxies=proxies)
logger.info(f'Download {url} via gevent completed')
def multi_threaded_download():
logger.info(f'Starting download via multi-thread')
threads = [download_via_thread(url) for url in urls]
for th in threads:
th.start()
logger.info(f'Downloading using {th.name}')
[th.join() for th in threads]
logger.info(f'Download via multi-thread completed')
def single_threaded_download():
greenlets = [gevent.spawn(download_via_gevent, url) for url in urls]
gevent.joinall(greenlets)
logger.debug('*** thread ***')
multi_threaded_download()
logger.debug('*** gevent ***')
single_threaded_download()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment