Skip to content

Instantly share code, notes, and snippets.

@rahulremanan
Last active July 31, 2021 13:40
Show Gist options
  • Save rahulremanan/57b62552015062886dad57e2a5c2cd95 to your computer and use it in GitHub Desktop.
Save rahulremanan/57b62552015062886dad57e2a5c2cd95 to your computer and use it in GitHub Desktop.
Managing memory utilization using Pickle
#===Import dependencies===
import gc
import pickle
import psutil
import numpy as np
import pandas as pd
#===Function to track memory usage===
def memory_utilization():
print('Current memory utilization: {}% ...'.format(psutil.virtual_memory().percent))
if __name__ == "__main__":
print(f'Memory utilization before creating the example Pandas dataframe: \n{memory_utilization()}')
#===Create a dataframe using random integers between 0 and 1000===
var=pd.DataFrame(np.random.randint(0,1000,size=(int(2.5e8),2)),columns=['var1','var2'])
print(f'Memory utilization after creating the example Pandas dataframe: \n{memory_utilization()}')
#==Create Pickle dump===
pickle.dump(var,open('var.pkl','wb'))
print(f'Memory utilization after creating the pickle dump: \n{memory_utilization()}')
#===Delete the unused variable from memory===
del var
_=gc.collect()
print(f'Memory utilization after deleting the variable from memory: \n{memory_utilization()}')
#===Restore the variable from the disk for future use===
var=pickle.load(open('var.pkl','rb'))
print(f'Memory utilization after re-loading the variable from the pickle dump to memory: \n{memory_utilization()}')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment