multiprocessing_counter.py

def counter(q, v, lock):
    while True:
        line = q.get()

        if line is None:
            return

        time.sleep(0.5)

        with lock:
            v.value += 1


def mp_counter(filename, num_processes=8):

    with mp.Manager() as manager:
        shared_value = manager.Value('i', 0)
        q = manager.Queue(num_processes)
        lock = manager.Lock()

        start = time.time()

        pool = []
        for i in range(num_processes):
            p = mp.Process(target=counter, args=(q, shared_value, lock))
            p.start()
            pool.append(p)

        with open(filename) as f:
            # poison pill
            lines = itertools.chain(f, (None,) * num_processes)
            for line in lines:
                q.put(line)

        for p in pool:
            p.join()

        print(shared_value.value)
        print('elapsed time: ', time.time() - start)
        
if __name__ == '__main__':
    filename = '/path/to/file'
    
    mp_counter(filename)