numpy file validation

numpy_file_validation.py

import argparse
import glob
import multiprocessing

import numpy
import tqdm

parser = argparse.ArgumentParser()
parser.add_argument('glob')
parser.add_argument('--processes', type=int)
args = parser.parse_args()


def try_load(p):
    try:
        numpy.load(p, mmap_mode='r')
    except:
        return p


if __name__ == '__main__':
    pool = multiprocessing.Pool(processes=args.processes)
    paths_input = list(glob.glob(args.glob))
    paths_invalid = list(tqdm.tqdm(pool.imap(try_load, paths_input), total=len(paths_input)))

    for path in paths_invalid:
        if path is None:
            continue

        print(path)