Skip to content

Instantly share code, notes, and snippets.

@yangyushi
Last active February 9, 2021 22:10
Show Gist options
  • Save yangyushi/0ad636490fa5c3b89d0f9d97ddfd6ea4 to your computer and use it in GitHub Desktop.
Save yangyushi/0ad636490fa5c3b89d0f9d97ddfd6ea4 to your computer and use it in GitHub Desktop.
this gist demonstrates the way to save a big numpy array as a binary string
import json
import base64
import numpy as np
from io import BytesIO
# save data to the binary string
a = np.random.uniform(0, 255, (1500, 1500)).astype(np.uint8)
f = BytesIO()
np.savez_compressed(f, data=a)
f.seek(0)
# convert bytes to a string
code = base64.b64encode(f.read()) # type: bytes
string = code.decode('utf-8') # type: str
f.close()
# dump the string to a json file
with open('test.json', 'w') as f:
json.dump({"data": string}, f, indent=" "*4)
# load the string from a json file
with open('test.json', 'r') as f:
data = json.load(f)["data"]
# convert sting to bytes
code = base64.b64decode(data)
f = BytesIO(code)
b = np.load(f, allow_pickle=True)['data']
f.close()
print(np.allclose(b, a))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment