Skip to content

Instantly share code, notes, and snippets.

@NP-chaonay
Last active April 24, 2020 16:33
Show Gist options
  • Save NP-chaonay/ab7e76b30b69010454c643d4265c83d9 to your computer and use it in GitHub Desktop.
Save NP-chaonay/ab7e76b30b69010454c643d4265c83d9 to your computer and use it in GitHub Desktop.
My simple implementation of Out-of-memory mechanism for Linux.
#!/usr/bin/python3
import glob,os,time,sys
import pandas as pd
# Program parameters
CHECKING_DELAY_TIME=20
LOW_MEM_ENSURE_DELAY_TIME=10
SIGTERM_DELAY_TIME=20
SYS_UNDERLOADING_DELAY_TIME=10
LOW_MEM_THRESHOLD_KIB=100000
# Template function for stdout
def print_template(msg):
localtime=time.localtime()
print('[{}-{}-{}_{}:{}:{}] : '.format(AddLeadZero(localtime[0],4),AddLeadZero(localtime[1],2),AddLeadZero(localtime[2],2),AddLeadZero(localtime[3],2),AddLeadZero(localtime[4],2),AddLeadZero(localtime[5],2))+msg)
# Checking wheather if system in low memory situation
def isLowMemSys():
lines=open('/proc/meminfo').read().splitlines()
for i in range(len(lines)):
lines[i]=lines[i].split(None,1)
lines[i][0]=lines[i][0][:-1]
if len(lines[i])!=2: lines[i].append(None)
lines=dict(lines)
return int(lines.get('MemAvailable')[:-3])<LOW_MEM_THRESHOLD_KIB
# Imported functions from my created library
# Template function for printing Python exception
def print_err(err,action='doing a operation',type_='Warning'):
print('['+type_+'] Exception occurred when '+action+'.')
print(' '+type(err).__name__+': '+str(err))
# Turns integer to string with specific leading zeroes
def AddLeadZero(num,digit,IgnoreDataManipulation=False,RaiseDataManipulationError=False,DigitMustAtLeastTwo=False):
"""Add leading the letters '0' to inputted integer 'num' according to defined 'digit' and return as string.
Required keyword arguments:
- num (int) : Integer (can be positive, zero, or negative)
- digit (int) : How much digits of number should be in returned string.
Optional keyword arguments:
- IgnoreDataManipulation (bool) : Avoid raising acceptable data manipulation error either in stderr message or in error exception form.
- RaiseDataManipulationError (bool) : Raise every data manipulation error as error exception.
- DigitMustAtLeastTwo (bool) : Raise warning or error if defined digit is less than 2.
Data manipulation error:
- Digit should be at least 2. (Ignore by default)
- Amount of defined digits is less than digits of number in inputted integer.
"""
if type(num) is not int or type(digit) is not int: raise TypeError('parameters \'num\', \'digit\' should be integer.')
if type(IgnoreDataManipulation) is not bool or type(RaiseDataManipulationError) is not bool or type(DigitMustAtLeastTwo) is not bool: raise TypeError('parameters \'IgnoreDataManipulation\', \'RaiseDataManipulationError\', and \'DigitMustAtLeastTwo\' should be boolean.')
if IgnoreDataManipulation: RaiseDataManipulationError=False
if digit<1: raise ValueError('Digit should be at least one.')
if digit<2 and DigitMustAtLeastTwo:
Error=ValueError('Amount of digits should be at least 2.')
if not IgnoreDataManipulation and not RaiseDataManipulationError: sys.stderr.write('Warning:\n Namespace \''+__name__+'\'\n'+type(Error).__name__+': '+str(Error)+'\n')
if RaiseDataManipulationError: raise Error
# Reuse variable 'digit'
if num>0:
num=str(num)
IsNegative=False
else:
num=str(abs(num))
IsNegative=True
digit=digit-len(num)
if digit>0:
for x in range(0,digit):
# Reuse variable 'num'
num='0'+num
if not IsNegative: return num
else: return '-'+num
elif digit==0:
if not IsNegative: return num
else: return '-'+num
else:
Error=ValueError('Defined digits amount is less than digits of number in inputted integer. It possibly means that some of used data has been manipulated incorrectly.')
if not IgnoreDataManipulation and not RaiseDataManipulationError: sys.stderr.write('Warning:\n Namespace \''+__name__+'\'\n'+type(Error).__name__+': '+str(Error)+'\n')
if RaiseDataManipulationError: raise Error
if not IsNegative: return num
else: return '-'+num
# Renice to -20 (Same as earlyoom)
try: os.nice(-20)
except PermissionError as err:
print_err(err,'nicing the process')
print('Current PID :',os.getpid())
while True:
if isLowMemSys():
print_template('Minimum resident memory threshold is reached.')
print_template('Waiting for '+str(LOW_MEM_ENSURE_DELAY_TIME)+' seconds to check if the system is really in low memory situation...')
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "System is running in low memory situation" "User OOM will take the action if this persist to continue after '+str(LOW_MEM_ENSURE_DELAY_TIME)+' seconds delay."')
time.sleep(LOW_MEM_ENSURE_DELAY_TIME)
if isLowMemSys():
pid_status_files=glob.glob('/proc/*/status')
del pid_status_files[0:2]
PidList=[]
NameList=[]
RssAnonList=[]
VmRSSList=[]
for file in pid_status_files:
try: lines=open(file).read().splitlines()
except: continue
for i in range(len(lines)):
lines[i]=lines[i].split(None,1)
lines[i][0]=lines[i][0][:-1]
if len(lines[i])!=2: lines[i].append(None)
lines=dict(lines)
val=lines.get('Pid')
if val!=None: PidList+=[int(val)]
else: PidList+=[None]
val=lines.get('Name')
if val!=None: NameList+=[val]
else: NameList+=[None]
val=lines.get('RssAnon')
if val!=None: RssAnonList+=[int(val[:-3])]
else: RssAnonList+=[None]
val=lines.get('VmRSS')
if val!=None: VmRSSList+=[int(val[:-3])]
else: VmRSSList+=[None]
process_table=pd.DataFrame({'Name':NameList,'VmRSS':VmRSSList,'RssAnon':RssAnonList},index=PidList,dtype='object')
process_table.index.name='PID'
i=process_table.RssAnon.astype('float').idxmax() # Choose memory usage querying method between RssAnon (Default Recommended) or VmRSS
print_template('The process (PID '+str(i)+') need to be killed to solve low memory situation. :')
print('\n'.join(['\t'+row for row in str(process_table.loc[i]).splitlines()]))
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "User OOM is taking the low memory action" "The one process is required to be killed to ensure the system stability. See the User OOM console output for details."')
try:
print_template('Sending signal SIGTERM for terminating the process '+str(i)+'...')
os.kill(i,15)
print_template('Waiting '+str(SIGTERM_DELAY_TIME)+' seconds for process to terminate...')
time.sleep(SIGTERM_DELAY_TIME)
except ProcessLookupError:
print_template('Waiting '+str(SYS_UNDERLOADING_DELAY_TIME)+' seconds for system to underloading...')
time.sleep(SYS_UNDERLOADING_DELAY_TIME); continue
except (PermissionError,OSError) as err:
print_err(err,'killing the process','Critical')
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "User OOM fails to take the low memory action" "The process that should be killed could not be killed by User OOM. See the User OOM console output for details."')
print_template('Waiting '+str(SYS_UNDERLOADING_DELAY_TIME)+' seconds for system to underloading...')
time.sleep(SYS_UNDERLOADING_DELAY_TIME); continue
try:
print_template('Sending signal SIGKILL for killing the process '+str(i)+'...')
os.kill(i,9)
print_template('Waiting '+str(SYS_UNDERLOADING_DELAY_TIME)+' seconds for system to underloading...')
time.sleep(SYS_UNDERLOADING_DELAY_TIME)
except ProcessLookupError: continue
except (PermissionError,OSError) as err:
print_err(err,'killing the process','Critical')
os.system('notify-send -u critical -i "/usr/share/icons/Humanity/devices/48/media-memory.svg" "User OOM fails to take the low memory action" "The process that should be killed could not be killed by User OOM. See the User OOM console output for details."')
continue
else: time.sleep(CHECKING_DELAY_TIME)
else: time.sleep(CHECKING_DELAY_TIME)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment