download.py

import threading
from contextlib import closing
import requests
from Queue import Queue


input_file = u"test_input.txt"


def on_download_complete(url, file_name):
    """
    Callback to be fired on the main thread when the download has
    completed.
    """
    print "{} completed!".format(file_name)


def on_download_error(url, error):
    """
    Callback to be fired on the main thread in case of error.
    """
    print "{} failed due to {}".format(url, error)


def download(url, local_name, msg_queue):
    """
    The download worker entrypoint.
    """
    try:
        with closing(requests.get(url, stream=True)) as response:
            if not response.ok:
                response.raise_for_status()
            with open(local_name, 'wb') as local_file:
                for chunk in response.iter_content(chunk_size=1024):
                    if chunk:
                        local_file.write(chunk)
    except Exception as e:
        msg_queue.put(lambda: on_download_error(url, e.message))
    else:
        msg_queue.put(lambda: on_download_complete(url, local_name))


if __name__ == "__main__":
    # a queue to receive events for the main thread
    msg_queue = Queue()

    # track the thread count against this baseline
    thread_count = threading.active_count()

    # open the input file, read the urls, for each clean up the
    # text, determine the local filname, and then kick off a worker
    with open(input_file, "r") as f:
        for url in f:
            clean_url = url.strip()
            short_url = '/'.join(clean_url.split('/')[:-1])
            local_name = clean_url.split('/')[-1]
            print "Downloading {} from {} ...".format(local_name, short_url)
            t = threading.Thread(target=download, args=(clean_url, local_name, msg_queue))
            t.start()

    # keep processing events in the message queue until the thread
    # count returns to the baseline
    while threading.active_count() > thread_count:
        event = msg_queue.get()
        event()

    print "All downloads completed!"

Thank you sir