prefetch.py

#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# $File: prefetch.py
# $Date: Tue Aug 26 18:12:01 2014 +0800
# $Author: Xinyu Zhou <zxytim[at]gmail[dot]com>

from multiprocessing import Pool
from collections import deque


def fetch_single_data(conf):
    # you can read from disk, or crawl from websites, etc...
    print 'fetch_single_data', conf
    return conf


def iter_data(data_conf, num_proc):
    pool = Pool(num_proc)
    queue = deque()
    for conf in data_conf:
        queue.append(pool.apply_async(fetch_single_data, [conf]))
        if len(queue) == num_proc + 1:
            r = queue.popleft()
            yield r.get()
    for r in queue:
        yield r.get()
    pool.close()
    pool.join()
    pool.terminate()


def main():
    for i in iter_data(range(100), 10):
        print i


if __name__ == '__main__':
    main()

# vim: foldmethod=marker