Pandas groupby apply multiprocessing #python #pandas

pandas_groupby_apply_multiprocessing.py

from joblib import Parallel, delayed
import multiprocessing
import pandas as pd
import time
def applyParallel(dfGrouped, func):
    retLst = Parallel(n_jobs=multiprocessing.cpu_count())(delayed(func)(group) for name, group in dfGrouped)
    return pd.concat(retLst)
def myfunc(df)
    return df
  
start_time = time.time()
res = applyParallel(df.groupby(['id']), myfunc)
print(time.time() - start_time)

Thanks!

if we want to parse som "args" to myfunc - how would we do that?