Watch a log file

log_watcher.py

from datetime import datetime, timedelta
from sys import argv, exit
import time
import signal

def usage():
    print '''
    python log.py filename

    - we will watch filename for events.
    - if the file is renamed, then we will attempt to reopen the original filename
    - if the file is truncated, then we will read from the beginning again

    '''
    exit(1)

def signal_handler(sig, frame):
    onExit()
    exit(0)

signal.signal(signal.SIGINT, signal_handler)

# Alert threshold
threshold=5

# Used to record file position
cur = 0

# Bucket_counter holds the number of events per hour
class Bucket_counter:

    def __init__(self, threshold):
        # Initialize to epoch
        self.last_time_bucket = datetime(1970, 1, 1, 0, 0, 0)
        self.total = -1
        self.threshold = threshold
        self.triggered = False

    def nextBucket(self, new_time_bucket):
        end_time = self.last_time_bucket - timedelta(seconds=1)
        # Do not print the first nextBucket() call for 1970
        if self.total >= 0:
            print "%s %s: %d" % (self.last_time_bucket, end_time, self.total)
        self.last_time_bucket = new_time_bucket
        self.total = 0
        self.triggered = False

    def add(self, event_time):

        # Round timestamp down to hour:
        # From: https://stackoverflow.com/questions/3463930/how-to-round-the-minute-of-a-datetime-object-python
        delta = timedelta(minutes=event_time.minute,
           seconds=event_time.second,
           microseconds=event_time.microsecond)
        event_time_bucket = event_time - delta

        # Check if we need to switch to the next hour
        if self.last_time_bucket != event_time_bucket:
            self.nextBucket(event_time_bucket)

        self.total = self.total + 1

        # Check if we have crossed the threshold
        if self.total >= self.threshold and not self.triggered:
            self.triggered = True
            print "Alert triggered! Threshold is %d, Current is %d" % (self.threshold, self.total)

    def done(self):
        self.nextBucket(datetime.now())

# COUNTER holds our total number of events per hour
COUNTER = Bucket_counter(threshold)

def inspectline(line):
    global COUNTER

    fields = line.split(' ')

    # skip partial lines
    if len(fields) < 8:
        return

    # parse values
    try:
        statuscode = int(fields[8])
        timefield = fields[3].replace("[", "")
        time = datetime.strptime(timefield, '%d/%b/%Y:%H:%M:%S')
    except ValueError, err:
        print str(err)
        return

    # record event
    if statuscode >= 500:
        COUNTER.add(time)

# Follow a file
# Found this example here: https://stackoverflow.com/questions/1475950/tail-f-in-python-with-no-time-sleep
# https://stackoverflow.com/questions/25537237/python-read-file-continuously-even-after-it-has-been-logrotated
def follow(filename):

    def is_truncated(f):
        global cur
        truncate = False
        cur = f.tell()
        f.seek(0,2)
        if f.tell() < cur:
            #print "Truncate detected"
            truncate = True
            f.seek(0,0)
        else:
            f.seek(cur,0)
        return truncate

    # assume file has rotated, if file length diffs from open filehandle's length
    # It will not return true, if the file "name" does not exist.
    def has_moved(name, handle):
        def eof(f):
            f.seek(0,2)
            return f.tell()

        moved = False
        original_pos = handle.tell()
        try:
            test = open(name, "r")
            if eof(test) != eof(handle):
                moved = True

            test.close()
        except IOError, err:
            #print err
            pass

        # Return to original position
        handle.seek(original_pos, 0)
        return moved

    filehandle = open(filename, "r")
    while True:
        line = filehandle.readline()
        if not line:
            if is_truncated(filehandle):
                return
            if has_moved(filename, filehandle):
                return
            time.sleep(0.1)    # Sleep briefly
            continue
        yield line

def watchfile(logfile):

    while True:
        input = follow(logfile)
        for logline in input:
            inspectline(logline)

def onExit():
    global COUNTER
    COUNTER.done()

def main():
    if len(argv) <= 1:
        usage()
    logfile = argv[1]
    watchfile(logfile)

main()