MultiThreadedFileLoader

import librosa
import threading
import numpy as np
import os

num_splits = 8
data_path = "ProcessedNumpys"
dataset = []

def add_to_arr(i, data_path, filenames, start, end):

    for j, filename in enumerate(filenames[start:end]):
        print(i, filename, j)
        arr = np.load(data_path + "\\" + filename)
        #wav, sr = librosa.load(data_path + "\\" + filename, sr=22050)
        dataset.append(arr)

        #np.save("ProcessedNumpys" +"\\"+ filename, wav, allow_pickle=True, fix_imports=True)




def load_dataset(data_path, splits):

    #dataset and threads

    threads = []

    #gets filenames and splits dataset up for processes
    filenames = os.listdir(data_path)
    split_size = len(filenames) // splits

    for i in range(splits):
        start = i * split_size
        end = None if i+1 == num_splits else (i+1) * split_size

        threads.append(
            threading.Thread(target=add_to_arr, args=(i, data_path, filenames,start,end))
        )

        threads[-1].start()

    # wait for all threads to finish
    for t in threads:
        t.join()

load_dataset(data_path, num_splits)
np.savez_compressed("drum_test{}", np.array(dataset))