Python Pretetcher

prefetcher.py

import threading
import queue
import time


class BackgroundGenerator(threading.Thread):
    def __init__(self, generator,max_prefetch = 100):
        threading.Thread.__init__(self)
        self.queue = queue.Queue(max_prefetch)
        self.generator = generator
        self.daemon = True
        self.start()

    def run(self):
        for item in self.generator:
            self.queue.put(item)
        self.queue.put(None)

    def next(self):
        next_item = self.queue.get()
        if next_item is None:
             raise StopIteration
        return next_item

    # Python 3 compatibility
    def __next__(self):
        return self.next()

    def __iter__(self):
        return self


def gen():
    for i in range(1000):
        time.sleep(1)
        yield i

generator = BackgroundGenerator(gen(), 10)

time.sleep(10)

now = time.time()
for i in range(10):
    print(generator.next())
print(time.time() - now)

now = time.time()
for i in range(10):
    print(next(gen()))
print(time.time() - now)