Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Batch Load Speed From Disk Benchmarking
def run_one_epoch(data, conn):
epoch_start = time.time()
for batch in data:
pass
epoch_end = time.time()
conn.send(round(epoch_end - epoch_start, 6))
def measure_data_load_speed(data, num_epochs):
"""
:param data: can be a DataSet or a DataLoader
"""
epoch_times = []
for i in range(num_epochs):
parent_conn, child_conn = Pipe()
p = Process(target=run_one_epoch, args=(data, child_conn))
p.start()
epoch_time = parent_conn.recv()
p.join()
epoch_times.append(epoch_time)
results = {'data': epoch_times,
'avg': round(stats.mean(epoch_times), 4),
'std': round(stats.stdev(epoch_times), 4)}
return results
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment