Skip to content

Instantly share code, notes, and snippets.

@ottonemo
Created March 13, 2018 15:48
Show Gist options
  • Save ottonemo/58e98fadab7ed474c75ce5242e972177 to your computer and use it in GitHub Desktop.
Save ottonemo/58e98fadab7ed474c75ce5242e972177 to your computer and use it in GitHub Desktop.
import torch
import pickle
ts = [torch.ones(10, 20) + n for n in range(4)]
diff = 0
with open('tensor.pickle', 'wb') as f:
for t in ts:
start = f.tell()
torch.save(t, f)
diff = f.tell() - start
print(f"start: {start}, end:", f.tell(), "diff", diff)
print("Loading...")
ts_read = []
with open('tensor.pickle', 'rb') as f:
for i in range(len(ts)):
f.seek(i * diff)
tl = torch.load(f)
ts_read.append(tl)
assert len(ts_read) == len(ts)
print(ts_read[-1])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment