Skip to content

Instantly share code, notes, and snippets.

@luispedro luispedro/pdio.py
Last active Mar 22, 2019

Embed
What would you like to do?
Save & load from a pandas DataFrame/Series
import numpy.lib
import numpy as np
import pandas as pd
import cPickle as pickle
def save_pandas(fname, data):
'''Save DataFrame or Series
Parameters
----------
fname : str
filename to use
data: Pandas DataFrame or Series
'''
np.save(open(fname, 'w'), data)
if len(data.shape) == 2:
meta = data.index,data.columns
elif len(data.shape) == 1:
meta = (data.index,)
else:
raise ValueError('save_pandas: Cannot save this type')
s = pickle.dumps(meta)
s = s.encode('string_escape')
with open(fname, 'a') as f:
f.seek(0, 2)
f.write(s)
def load_pandas(fname, mmap_mode='r'):
'''Load DataFrame or Series
Parameters
----------
fname : str
filename
mmap_mode : str, optional
Same as numpy.load option
'''
values = np.load(fname, mmap_mode=mmap_mode)
with open(fname) as f:
numpy.lib.format.read_magic(f)
numpy.lib.format.read_array_header_1_0(f)
f.seek(values.dtype.alignment*values.size, 1)
meta = pickle.loads(f.readline().decode('string_escape'))
if len(meta) == 2:
return pd.DataFrame(values, index=meta[0], columns=meta[1])
elif len(meta) == 1:
return pd.Series(values, index=meta[0])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.