manager.py

import multiprocessing
import threading
import time
from Queue import Queue


class Sequence():
    def __init__(self, my_list):
        self.my_list = my_list

    def __getitem__(self, item):
        return self.my_list[item]

    def __len__(self):
        return len(self.my_list)


def get_item(seq, idx):
    return seq[idx]


def main():
    files = ['test_string'] * 100000
    nb_files = min(len(list(files)), 100)

    stuff = ((x, [1, 3, 4]) for x in files)
    seq = Sequence(list(stuff))
    manager2 = multiprocessing.Manager()
    stuff = ((x, [1, 3, 4]) for x in files)
    ls = manager2.list(list(stuff))
    seq2 = Sequence(ls)

    # Test time!
    x = multiprocessing.Pool(5)
    qu = Queue(10)

    def run(qu):
        for i in range(nb_files):
            acc = qu.get(block=True).get()

    th = threading.Thread(target=run, args=(qu,))
    th.daemon = True
    th.start()
    st = time.time()
    for i in range(nb_files):
        qu.put(x.apply_async(func=get_item, args=(seq2, i,)), block=True)

    th.join()
    print("Took Manager list", time.time() - st)

    th = threading.Thread(target=run, args=(qu,))
    th.daemon = True
    th.start()
    st = time.time()
    for i in range(nb_files):
        qu.put(x.apply_async(func=get_item, args=(seq, i,)), block=True)

    th.join()
    print("Took list", time.time() - st)


main()