Last active
December 9, 2015 20:06
-
-
Save dmitriy-serdyuk/daca0067b0c18a2078d1 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import base64 | |
import json | |
import numpy as np | |
import cPickle | |
import pandas | |
class NumpyEncoder(json.JSONEncoder): | |
def default(self, obj): | |
"""If input object is an ndarray it will be converted into a dict | |
holding dtype, shape and the data, base64 encoded. | |
""" | |
if isinstance(obj, np.ndarray): | |
if obj.flags['C_CONTIGUOUS']: | |
obj_data = obj.data | |
else: | |
cont_obj = np.ascontiguousarray(obj) | |
assert(cont_obj.flags['C_CONTIGUOUS']) | |
obj_data = cont_obj.data | |
data_b64 = base64.b64encode(obj_data) | |
return dict(__ndarray__=data_b64, | |
dtype=str(obj.dtype), | |
shape=obj.shape) | |
# Let the base class default method raise the TypeError | |
return json.JSONEncoder(self, obj) | |
def json_numpy_obj_hook(dct): | |
"""Decodes a previously encoded numpy ndarray with proper shape and dtype. | |
:param dct: (dict) json encoded ndarray | |
:return: (ndarray) if input was an encoded ndarray | |
""" | |
if isinstance(dct, dict) and '__ndarray__' in dct: | |
data = base64.b64decode(dct['__ndarray__']) | |
return np.frombuffer(data, dct['dtype']).reshape(dct['shape']) | |
return dct | |
filename = 'main_log' | |
with open(filename + '.zip', 'r') as f: | |
log = cPickle.load(f) | |
df = pandas.DataFrame.from_dict(log, orient='index') | |
with open(filename + '.json', 'w') as f: | |
for i, row in df.iterrows(): | |
row = json.dumps(dict(row.dropna()), cls=NumpyEncoder) | |
f.write(row + '\n') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment