ApacheLogster Log Parser

ApacheLogster.py

###  A modified sample logster parser file that can be used to count the number
###  of response codes found in an Apache access log.
###
###  For example:
###  sudo ./logster --dry-run --output=ganglia ApacheLogster /var/log/httpd/access_log
###
###
###  Copyright 2011, Etsy, Inc.
###
###  This file is part of Logster.
###
###  Logster is free software: you can redistribute it and/or modify
###  it under the terms of the GNU General Public License as published by
###  the Free Software Foundation, either version 3 of the License, or
###  (at your option) any later version.
###
###  Logster is distributed in the hope that it will be useful,
###  but WITHOUT ANY WARRANTY; without even the implied warranty of
###  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
###  GNU General Public License for more details.
###
###  You should have received a copy of the GNU General Public License
###  along with Logster. If not, see <http://www.gnu.org/licenses/>.
###

import time
import re

from logster_helper import MetricObject, LogsterParser
from logster_helper import LogsterParsingException

class ApacheLogster(LogsterParser):

    def __init__(self, option_string=None):
        '''Initialize any data structures or variables needed for keeping track
        of the tasty bits we find in the log we are parsing.'''
        self.machines = {}

        # Regular expression for matching lines we are interested in, and capturing
        # fields from the line (in this case, http_status_code).
        self.reg = re.compile('\S{3} \d{02} \d{2}.\d{2}.\d{2} (?P<hostname>[\S]+).*HTTP/1.\d\" (?P<http_status_code>\d{3}) .*')

    def parse_line(self, line):
        '''This function should digest the contents of one line at a time, updating
        object's state variables. Takes a single argument, the line to be parsed.'''
        try:
            # Apply regular expression to each line and extract interesting bits.
            regMatch = self.reg.match(line)
            if regMatch:
                linebits = regMatch.groupdict()
                status = int(linebits['http_status_code'])
                machine = str(linebits['hostname'])
                if machine not in self.machines.keys():
                    self.machines[machine] = {'1xx':0,'2xx':0,'3xx':0,'4xx':0,'5xx':0}
                if (status < 200):
                    self.machines[machine]['1xx'] += 1
                elif (status < 300):
                    self.machines[machine]['2xx'] += 1
                elif (status < 400):
                    self.machines[machine]['3xx'] += 1
                elif (status < 500):
                    self.machines[machine]['4xx'] += 1
                else:
                    self.machines[machine]['5xx'] += 1
            else:
                raise LogsterParsingException, "regmatch failed to match"
        except Exception, e:
            raise LogsterParsingException, "regmatch or contents failed with %s on %s" % (e,line)

    def get_state(self, duration):
        '''Run any necessary calculations on the data collected from the logs
        and return a list of metric objects.'''
        self.duration = duration
        return_list = []
        for key in self.machines.keys():
            # Return a list of metrics objects
            return_list.append( MetricObject("web%s.http_1xx" % key, (self.machines[key]['1xx'] / self.duration), "Responses per sec") )
            return_list.append( MetricObject("web%s.http_2xx" % key, (self.machines[key]['2xx'] / self.duration), "Responses per sec") )
            return_list.append( MetricObject("web%s.http_3xx" % key, (self.machines[key]['3xx'] / self.duration), "Responses per sec") )
            return_list.append( MetricObject("web%s.http_4xx" % key, (self.machines[key]['4xx'] / self.duration), "Responses per sec") )
            return_list.append( MetricObject("web%s.http_5xx" % key, (self.machines[key]['5xx'] / self.duration), "Responses per sec") )
        return return_list

On CentOS 6.5, the most recent version of logster fails with:
Traceback (most recent call last):
File "/usr/bin/logster", line 443, in
main()
File "/usr/bin/logster", line 358, in main
module = import(module_name, globals(), locals(), [parser_name])
File "/usr/lib/python2.6/site-packages/logster/parsers/ApacheLogster.py", line 29, in
from logster_helper import MetricObject, LogsterParser
ImportError: No module named logster_helper

I had to change the following two lines:
from logster.logster_helper import MetricObject, LogsterParser
from logster.logster_helper import LogsterParsingException