Last active
July 31, 2021 13:40
-
-
Save rahulremanan/57b62552015062886dad57e2a5c2cd95 to your computer and use it in GitHub Desktop.
Managing memory utilization using Pickle
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#===Import dependencies=== | |
import gc | |
import pickle | |
import psutil | |
import numpy as np | |
import pandas as pd | |
#===Function to track memory usage=== | |
def memory_utilization(): | |
print('Current memory utilization: {}% ...'.format(psutil.virtual_memory().percent)) | |
if __name__ == "__main__": | |
print(f'Memory utilization before creating the example Pandas dataframe: \n{memory_utilization()}') | |
#===Create a dataframe using random integers between 0 and 1000=== | |
var=pd.DataFrame(np.random.randint(0,1000,size=(int(2.5e8),2)),columns=['var1','var2']) | |
print(f'Memory utilization after creating the example Pandas dataframe: \n{memory_utilization()}') | |
#==Create Pickle dump=== | |
pickle.dump(var,open('var.pkl','wb')) | |
print(f'Memory utilization after creating the pickle dump: \n{memory_utilization()}') | |
#===Delete the unused variable from memory=== | |
del var | |
_=gc.collect() | |
print(f'Memory utilization after deleting the variable from memory: \n{memory_utilization()}') | |
#===Restore the variable from the disk for future use=== | |
var=pickle.load(open('var.pkl','rb')) | |
print(f'Memory utilization after re-loading the variable from the pickle dump to memory: \n{memory_utilization()}') |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment