Parallel copy

parallel_copy.py

from pathlib import Path
import shutil
from tqdm import tqdm
from joblib import Parallel, delayed


def pcopy(dir_in, dir_out, files_list, processes=8):
    dir_in = Path(dir_in)
    dir_out = Path(dir_out)
    copy_list = [
        (dir_in / file, dir_out / file)
        for file in files_list
    ]
    def cp(pin, pout):
        pout.parent.mkdir(parents=True, exist_ok=True)
        shutil.copy(pin, pout)
    Parallel(processed)(delayed(cp)(pin, pout) for pin, pout in tqdm(copy_list))