Skip to content

Instantly share code, notes, and snippets.

@Markbnj
Created April 11, 2016 19:53
Show Gist options
  • Save Markbnj/5839e1567704cbadca9e19ed16b9e00b to your computer and use it in GitHub Desktop.
Save Markbnj/5839e1567704cbadca9e19ed16b9e00b to your computer and use it in GitHub Desktop.
import threading
from contextlib import closing
import requests
from Queue import Queue
input_file = u"test_input.txt"
def on_download_complete(url, file_name):
"""
Callback to be fired on the main thread when the download has
completed.
"""
print "{} completed!".format(file_name)
def on_download_error(url, error):
"""
Callback to be fired on the main thread in case of error.
"""
print "{} failed due to {}".format(url, error)
def download(url, local_name, msg_queue):
"""
The download worker entrypoint.
"""
try:
with closing(requests.get(url, stream=True)) as response:
if not response.ok:
response.raise_for_status()
with open(local_name, 'wb') as local_file:
for chunk in response.iter_content(chunk_size=1024):
if chunk:
local_file.write(chunk)
except Exception as e:
msg_queue.put(lambda: on_download_error(url, e.message))
else:
msg_queue.put(lambda: on_download_complete(url, local_name))
if __name__ == "__main__":
# a queue to receive events for the main thread
msg_queue = Queue()
# track the thread count against this baseline
thread_count = threading.active_count()
# open the input file, read the urls, for each clean up the
# text, determine the local filname, and then kick off a worker
with open(input_file, "r") as f:
for url in f:
clean_url = url.strip()
short_url = '/'.join(clean_url.split('/')[:-1])
local_name = clean_url.split('/')[-1]
print "Downloading {} from {} ...".format(local_name, short_url)
t = threading.Thread(target=download, args=(clean_url, local_name, msg_queue))
t.start()
# keep processing events in the message queue until the thread
# count returns to the baseline
while threading.active_count() > thread_count:
event = msg_queue.get()
event()
print "All downloads completed!"
@Tomitin
Copy link

Tomitin commented May 7, 2021

Thank you sir

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment