gistfile1.txt

import threading 
import random

...
...
...

    def download_logs(self):
        """
        Downloads logs from S3 using Boto.
        """
        if self.aws_keys:
            conn = S3Connection(*self.aws_keys)
        else:
            conn = S3Connection()

        files = []
        mybucket = conn.get_bucket(self.input_bucket)
        with tempdir.TempDir() as directory:
            for item in mybucket.list(prefix=self.input_prefix):
                local_file = os.path.join(directory, item.key.split("/")[-1])
                logger.debug("Downloading %s to %s" % (item.key, local_file))
                thread = threading.Thread(target=item.get_contents_to_filename, args=(local_file,))
                thread.start()
                files.append((thread,local_file))

            elms = range(len(files))
            elemslen = len(elms)
            while elemslen:
                curr = random.choice(elms)
                thread, file  = files[curr]
                if not thread.is_alive():
                    yield file
                    elms.remove(curr)
                    elemslen -= 1