Simple example of multiprocessing

loader-trainer-multiprocess.py

import torch.multiprocessing as tmp

N_MINIBATCHES = 10

def load_minibatches(q):
    for i in range(N_MINIBATCHES):
        # load minibatch i
        q.put(minibatch)
    q.close()

def train_neural_network(q):
    for i in range(N_MINIBATCHES):
        minibatch = q.get()
        # train with minibatch i

def main():
    q = tmp.Queue(maxsize=1)
    loader = tmp.Process(load_minibatches, args=(q,))
    loader.start()
    trainer = tmp.Process(train_neural_network, args=(q,))
    trainer.start()
    loader.join()
    trainer.join()