thenomemac

{"lastUpload":"2019-06-10T03:11:38.927Z","extensionVersion":"v3.2.9"}

thenomemac

# By Josiah Olson
# Context manager to generate batches in the background via a process pool
# Returns float32 minibatch tensors and metadata using numpy sharedmem to avoid 
# the thread blocking compute time needed to de-pickle large minibatches passed 
# through multiprocessing.Queue: ~0.3 sec for 1gb minibatch
# This context manager will bring blocking time to ~300 microseconds per minibatch
# Usage:
#
# def batchFxn(seed, sharedArr):
#    .... # generate iterable of float32 tensors in minibatch: batchArrs

thenomemac

# Modified 2016-06-30 by Josiah Olson to add python3 support
# Context manager to generate batches in the background via a process pool
# Usage:
#
# def batch(seed):
#    .... # generate minibatch
#    return minibatch
#
# with BatchGenCM(batch) as bg:
#    minibatch = next(bg)

thenomemac

# Modified by Josiah Olson to add python3 support
# Context manager to generate batches in the background via a process pool
# Returns minibatch of fixed shape as shared memory to avoid de-pickling compute time
# Usage:
#
# def batch(seed, arr):
#    .... # generate minibatch
#    arr[:] = minibatch.ravel()
#    return minibatch.shape()
#

thenomemac

# Modified 2016-06-30 by Josiah Olson to add python3 support
# Context manager to generate batches in the background via a process pool
# Usage:
#
# def batch(seed):
#    .... # generate minibatch
#    return minibatch
#
# with BatchGenCM(batch) as bg:
#    minibatch = next(bg)