Small Python 3 script to show how to use multiprocessing for parallel processing of data

sample_multiprocessing.py

import pandas as pd
import numpy as np
import multiprocessing
from multiprocessing import Pool

num_processes = multiprocessing.cpu_count()
# on some systems, these next 2 lines will give better count for CPU intensive tasks
# import psutil
# num_processes = psutil.cpu_count(logical=False)
num_partitions =   num_processes * 2 #smaller batches to get more frequent status updates

df = pd.DataFrame(np.random.randint(0,100,size=(100, 4)), columns=list('ABCD'))

# put your code to parallelize processing of partitions of df here
def process_df(my_df):
    print("received df", my_df.shape)

with Pool(processes=num_processes) as pool:
    df_split = np.array_split(df, num_partitions)
    pool.map(process_df, df_split)