Last active
April 24, 2020 16:33
-
-
Save NP-chaonay/ab7e76b30b69010454c643d4265c83d9 to your computer and use it in GitHub Desktop.
My simple implementation of Out-of-memory mechanism for Linux.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
import glob,os,time,sys | |
import pandas as pd | |
# Program parameters | |
CHECKING_DELAY_TIME=20 | |
LOW_MEM_ENSURE_DELAY_TIME=10 | |
SIGTERM_DELAY_TIME=20 | |
SYS_UNDERLOADING_DELAY_TIME=10 | |
LOW_MEM_THRESHOLD_KIB=100000 | |
# Template function for stdout | |
def print_template(msg): | |
localtime=time.localtime() | |
print('[{}-{}-{}_{}:{}:{}] : '.format(AddLeadZero(localtime[0],4),AddLeadZero(localtime[1],2),AddLeadZero(localtime[2],2),AddLeadZero(localtime[3],2),AddLeadZero(localtime[4],2),AddLeadZero(localtime[5],2))+msg) | |
# Checking wheather if system in low memory situation | |
def isLowMemSys(): | |
lines=open('/proc/meminfo').read().splitlines() | |
for i in range(len(lines)): | |
lines[i]=lines[i].split(None,1) | |
lines[i][0]=lines[i][0][:-1] | |
if len(lines[i])!=2: lines[i].append(None) | |
lines=dict(lines) | |
return int(lines.get('MemAvailable')[:-3])<LOW_MEM_THRESHOLD_KIB | |
# Imported functions from my created library | |
# Template function for printing Python exception | |
def print_err(err,action='doing a operation',type_='Warning'): | |
print('['+type_+'] Exception occurred when '+action+'.') | |
print(' '+type(err).__name__+': '+str(err)) | |
# Turns integer to string with specific leading zeroes | |
def AddLeadZero(num,digit,IgnoreDataManipulation=False,RaiseDataManipulationError=False,DigitMustAtLeastTwo=False): | |
"""Add leading the letters '0' to inputted integer 'num' according to defined 'digit' and return as string. | |
Required keyword arguments: | |
- num (int) : Integer (can be positive, zero, or negative) | |
- digit (int) : How much digits of number should be in returned string. | |
Optional keyword arguments: | |
- IgnoreDataManipulation (bool) : Avoid raising acceptable data manipulation error either in stderr message or in error exception form. | |
- RaiseDataManipulationError (bool) : Raise every data manipulation error as error exception. | |
- DigitMustAtLeastTwo (bool) : Raise warning or error if defined digit is less than 2. | |
Data manipulation error: | |
- Digit should be at least 2. (Ignore by default) | |
- Amount of defined digits is less than digits of number in inputted integer. | |
""" | |
if type(num) is not int or type(digit) is not int: raise TypeError('parameters \'num\', \'digit\' should be integer.') | |
if type(IgnoreDataManipulation) is not bool or type(RaiseDataManipulationError) is not bool or type(DigitMustAtLeastTwo) is not bool: raise TypeError('parameters \'IgnoreDataManipulation\', \'RaiseDataManipulationError\', and \'DigitMustAtLeastTwo\' should be boolean.') | |
if IgnoreDataManipulation: RaiseDataManipulationError=False | |
if digit<1: raise ValueError('Digit should be at least one.') | |
if digit<2 and DigitMustAtLeastTwo: | |
Error=ValueError('Amount of digits should be at least 2.') | |
if not IgnoreDataManipulation and not RaiseDataManipulationError: sys.stderr.write('Warning:\n Namespace \''+__name__+'\'\n'+type(Error).__name__+': '+str(Error)+'\n') | |
if RaiseDataManipulationError: raise Error | |
# Reuse variable 'digit' | |
if num>0: | |
num=str(num) | |
IsNegative=False | |
else: | |
num=str(abs(num)) | |
IsNegative=True | |
digit=digit-len(num) | |
if digit>0: | |
for x in range(0,digit): | |
# Reuse variable 'num' | |
num='0'+num | |
if not IsNegative: return num | |
else: return '-'+num | |
elif digit==0: | |
if not IsNegative: return num | |
else: return '-'+num | |
else: | |
Error=ValueError('Defined digits amount is less than digits of number in inputted integer. It possibly means that some of used data has been manipulated incorrectly.') | |
if not IgnoreDataManipulation and not RaiseDataManipulationError: sys.stderr.write('Warning:\n Namespace \''+__name__+'\'\n'+type(Error).__name__+': '+str(Error)+'\n') | |
if RaiseDataManipulationError: raise Error | |
if not IsNegative: return num | |
else: return '-'+num | |
# Renice to -20 (Same as earlyoom) | |
try: os.nice(-20) | |
except PermissionError as err: | |
print_err(err,'nicing the process') | |
print('Current PID :',os.getpid()) | |
while True: | |
if isLowMemSys(): | |
print_template('Minimum resident memory threshold is reached.') | |
print_template('Waiting for '+str(LOW_MEM_ENSURE_DELAY_TIME)+' seconds to check if the system is really in low memory situation...') | |
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "System is running in low memory situation" "User OOM will take the action if this persist to continue after '+str(LOW_MEM_ENSURE_DELAY_TIME)+' seconds delay."') | |
time.sleep(LOW_MEM_ENSURE_DELAY_TIME) | |
if isLowMemSys(): | |
pid_status_files=glob.glob('/proc/*/status') | |
del pid_status_files[0:2] | |
PidList=[] | |
NameList=[] | |
RssAnonList=[] | |
VmRSSList=[] | |
for file in pid_status_files: | |
try: lines=open(file).read().splitlines() | |
except: continue | |
for i in range(len(lines)): | |
lines[i]=lines[i].split(None,1) | |
lines[i][0]=lines[i][0][:-1] | |
if len(lines[i])!=2: lines[i].append(None) | |
lines=dict(lines) | |
val=lines.get('Pid') | |
if val!=None: PidList+=[int(val)] | |
else: PidList+=[None] | |
val=lines.get('Name') | |
if val!=None: NameList+=[val] | |
else: NameList+=[None] | |
val=lines.get('RssAnon') | |
if val!=None: RssAnonList+=[int(val[:-3])] | |
else: RssAnonList+=[None] | |
val=lines.get('VmRSS') | |
if val!=None: VmRSSList+=[int(val[:-3])] | |
else: VmRSSList+=[None] | |
process_table=pd.DataFrame({'Name':NameList,'VmRSS':VmRSSList,'RssAnon':RssAnonList},index=PidList,dtype='object') | |
process_table.index.name='PID' | |
i=process_table.RssAnon.astype('float').idxmax() # Choose memory usage querying method between RssAnon (Default Recommended) or VmRSS | |
print_template('The process (PID '+str(i)+') need to be killed to solve low memory situation. :') | |
print('\n'.join(['\t'+row for row in str(process_table.loc[i]).splitlines()])) | |
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "User OOM is taking the low memory action" "The one process is required to be killed to ensure the system stability. See the User OOM console output for details."') | |
try: | |
print_template('Sending signal SIGTERM for terminating the process '+str(i)+'...') | |
os.kill(i,15) | |
print_template('Waiting '+str(SIGTERM_DELAY_TIME)+' seconds for process to terminate...') | |
time.sleep(SIGTERM_DELAY_TIME) | |
except ProcessLookupError: | |
print_template('Waiting '+str(SYS_UNDERLOADING_DELAY_TIME)+' seconds for system to underloading...') | |
time.sleep(SYS_UNDERLOADING_DELAY_TIME); continue | |
except (PermissionError,OSError) as err: | |
print_err(err,'killing the process','Critical') | |
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "User OOM fails to take the low memory action" "The process that should be killed could not be killed by User OOM. See the User OOM console output for details."') | |
print_template('Waiting '+str(SYS_UNDERLOADING_DELAY_TIME)+' seconds for system to underloading...') | |
time.sleep(SYS_UNDERLOADING_DELAY_TIME); continue | |
try: | |
print_template('Sending signal SIGKILL for killing the process '+str(i)+'...') | |
os.kill(i,9) | |
print_template('Waiting '+str(SYS_UNDERLOADING_DELAY_TIME)+' seconds for system to underloading...') | |
time.sleep(SYS_UNDERLOADING_DELAY_TIME) | |
except ProcessLookupError: continue | |
except (PermissionError,OSError) as err: | |
print_err(err,'killing the process','Critical') | |
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "User OOM fails to take the low memory action" "The process that should be killed could not be killed by User OOM. See the User OOM console output for details."') | |
continue | |
else: time.sleep(CHECKING_DELAY_TIME) | |
else: time.sleep(CHECKING_DELAY_TIME) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment