batch_multiprocess.py

batch_multiprocess.py

def batch_multiprocess(inputs):
    """
    Compute multiprocess with one batch per process.
    """
    len_inputs = len(inputs)
    n_batches = multiprocessing.cpu_count()
    batch_size = len_inputs // n_batches

    batches = []
    for i in range(n_batches):
        start = batch_size * i
        end = (i + 1) * batch_size if i != n_batches - 1 else len_inputs
        batch = inputs[start:end]
        batches.append(batch)

    return Parallel(n_jobs=-1)(
        delayed(single_thread.__wrapped__)(batch) for batch in batches
    )