Parse a logfile in realtime for an email storm

email_threshold_check.py

#! /usr/bin/env python

'''

    email_throttle.py
    Matt Jones caffeinatedengineering@gmail.com
    Created 04.23.14
    Last Update 05.05.14

    Notes:


    Usage: ./email_throttle


    ToDo:


'''

import time
import subprocess
import sys
import os
sys.path.append('/usr/local/icingadata/store/icinga_service_checks/modules/python26/psutil-2.1.0-py2.6-linux-x86_64.egg')
import psutil


log_file = '/var/log/icinga/icinga.log'
email_threshold = 25
f = open(log_file, 'r')

def py_tail():
      '''
        py_tail
        takes: nothing
        returns: the current line
        function: get the newest line in a file

        This will function the same as the unix command 'tail -f'
    '''

  # get the current position of the cursor as set in the main loop
  where = f.tell()
  line = ''
  tail  = f.readline()
  # this will help prevent partial lines
  if not tail:
    time.sleep(0.1)
    f.seek(where)
  else:
    line += tail
  return line

def check_email(end_time):
      '''
        check_email
        takes: end_time
        returns: bool
        function: see if the current line is a notification command

        This will cheeck the current line to see if it is a
        notification command, and return 'True' if so.
    '''
  
  email_counter = 0
  while time.time() <= end_time:
    if 'NOTIFICATION' in py_tail():
      email_counter += 1
      if email_counter >= email_threshold:
        # email monitoring and the goc
        sleep(10)
        result = subprocess.Popen(['service', 'sendmail', 'stop'], shell=True, stdout=subprocess.PIPE)
        if check_maillog == True:
          # stop the service
        else:
          result = subprocess.Popen(['service', 'ido2db', 'stop'], shell=True, stdout=subprocess.PIPE)
          result = subprocess.Popen(['service', 'icinga', 'stop'], shell=True, stdout=subprocess.PIPE)
          if check_maillog == True:
            # stop the service
          else:
            result = subprocess.Popen(['shutdown', '-h', 'now'], shell=True, stdout=subprocess.PIPE)
    else:
      # return to the start of the while loop
      continue
  return True

def check_service():
      '''
        check_service
        takes: nothing
        returns: a bool
        function: check to make sure the Icinga service is running

        This will check to make sure the Icinga service is currently running,
        if so it will return 'True'
      '''
  service_pid = 00

  # Get the pid
  for x in psutil.process_iter():
    if x.name() == 'icinga':
      service_pid = x.pid
    else:
      pass

  # Check to make sure the service is present
  if service_pid == 00:
    print "the service is not running"
    return False
  else:
    return True

def check_maillog():
  '''
        check_maillog
        takes: nothing
        returns: a bool
        function: check to make sure all outgoing email has stopped

        This will check to make sure all outgoing email has stopped,
        if so it will return 'True'
      '''
  
  mail_log = '/var/log/maillog'
  f = open('mail_log', r)
  results = os.stat(mail_log)
  size_1 = st_results[6]
  sleep(10)
  results = os.stat(mail_log)
  size_2 = st_results[6]
  if size_2 != size_1:
    return False
  else:
   return True

def main():

  try:
    try:
      if os.fork() > 0:
        sys.exit(0)
    except OSError, e:
      sys.stderr.write('fork #1 failed" (%d) %s\n' % (e.errno, e.strerror))
      sys.exit(1)

    os.setsid()
    os.chdir('/home/majones')
    os.umask(0)

    try:
      pid = os.fork()
      if pid > 0:
        fpid = open('pidfile', 'wb')
        fpid.write(str(pid))
        fpid.close()
        sys.exit(0)
    except OSError, e:
      sys.stderr.write('fork #2 failed" (%d) %s\n' % (e.errno, e.strerror))
      sys.exit(1)

    si = open('/dev/null', 'r')
    so = open('out_log', 'a+', 0)
    se = open('err_log', 'a+', 0)
  
    os.dup2(si.fileno(), sys.stdin.fileno())
    os.dup2(so.fileno(), sys.stdout.fileno())
    os.dup2(se.fileno(), sys.stderr.fileno())
  
  except Exception, e:
    sys.stderr.write(str(e))

  # number of times to check if Icinga is running before stopping the service
  check_count = 0
  check_count_threshold = 3

  while check_count < check _count_threshold:
    check_count += 1
    while check_service()  == True:
      start_time = time.time()
      end_time = start_time + 60
    
      #Find the size of the file and move to the end
      st_results = os.stat(log_file)
      st_size = st_results[6]
      f.seek(st_size)

      check_email(end_time)
      continue
    # sleep for 1 minute then recheck the service
    sleep(60)
    continue

  # make the service stop and exit cleanly

  if __name__ == "__main__":
    main()