多进程读取多个数据文件

multiple_read.py

from glob import glob
import multiprocessing

from tqdm import tqdm
import pandas as pd

def json_reader(fname):
    df = pd.read_json(fname, lines=True)
    return df

def parquet_reader(fname):
    part_df = pd.read_parquet(fname)  # .query('has_exposure==1')
    return part_df

# filenames = glob('data/dt=2020-04-01/part*')
pool = multiprocessing.Pool(10)
callbacks = []
pbar = tqdm(total=len(filenames))
for fname in filenames:
    callbacks.append(pool.apply_async(parquet_reader, args=(fname, ), callback=lambda _: pbar.update(1)))
pool.close()
pool.join()
df = pd.concat([cb.get() for cb in callbacks], ignore_index=True, sort=True, copy=False)
pool.terminate()