Skip to content

Instantly share code, notes, and snippets.

Created April 11, 2016 19:53
What would you like to do?
import threading
from contextlib import closing
import requests
from Queue import Queue
input_file = u"test_input.txt"
def on_download_complete(url, file_name):
Callback to be fired on the main thread when the download has
print "{} completed!".format(file_name)
def on_download_error(url, error):
Callback to be fired on the main thread in case of error.
print "{} failed due to {}".format(url, error)
def download(url, local_name, msg_queue):
The download worker entrypoint.
with closing(requests.get(url, stream=True)) as response:
if not response.ok:
with open(local_name, 'wb') as local_file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
except Exception as e:
msg_queue.put(lambda: on_download_error(url, e.message))
msg_queue.put(lambda: on_download_complete(url, local_name))
if __name__ == "__main__":
# a queue to receive events for the main thread
msg_queue = Queue()
# track the thread count against this baseline
thread_count = threading.active_count()
# open the input file, read the urls, for each clean up the
# text, determine the local filname, and then kick off a worker
with open(input_file, "r") as f:
for url in f:
clean_url = url.strip()
short_url = '/'.join(clean_url.split('/')[:-1])
local_name = clean_url.split('/')[-1]
print "Downloading {} from {} ...".format(local_name, short_url)
t = threading.Thread(target=download, args=(clean_url, local_name, msg_queue))
# keep processing events in the message queue until the thread
# count returns to the baseline
while threading.active_count() > thread_count:
event = msg_queue.get()
print "All downloads completed!"
Copy link

Tomitin commented May 7, 2021

Thank you sir

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment