wc -l

wc-mp-pool.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import multiprocessing
import os
import sys
import time
import threading


def get_chunk_line_count((name, start, stop, blocksize)):
    left = stop - start

    def blocks(f, left):
        while left > 0:
            b = f.read(min(left, blocksize))
            if b:
                yield b
            else:
                break
            left -= len(b)

    with open(name, 'r') as f:
        f.seek(start)
        return sum(bl.count('\n') for bl in blocks(f, left))


def get_file_offset_ranges(name, blocksize=65536, m=1):
    fsize = os.stat(name).st_size
    chunksize = (fsize // multiprocessing.cpu_count()) * m
    n = fsize // chunksize

    ranges = []
    for i in range(0, n * chunksize, chunksize):
        ranges.append((name, i, i + chunksize, blocksize))
    if fsize % chunksize != 0:
        ranges.append((name, ranges[-1][2], fsize, blocksize))

    return ranges


def wc_mp_pool(name, blocksize=65536):
    ranges = get_file_offset_ranges(name, blocksize)

    pool = multiprocessing.Pool(processes=len(ranges))
    pool_outputs = pool.map(get_chunk_line_count, ranges)
    pool.close()
    pool.join()

    return sum(pool_outputs)


print(wc_mp_pool(sys.argv[1]))

Consider changing the function argument of get_chunk_line_count() in:

def get_chunk_line_count(ranges):
    name, start, stop, blocksize = ranges
    left = stop - start

    def blocks(f, left):
        while left > 0:
            b = f.read(min(left, blocksize))
            if b:
                yield b
            else:
                break
            left -= len(b)

    with open(name, 'r') as f:
        f.seek(start)
        return sum(bl.count('\n') for bl in blocks(f, left))

Thank you for sharing.

Adriano