Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
ApacheLogster Log Parser
### A modified sample logster parser file that can be used to count the number
### of response codes found in an Apache access log.
###
### For example:
### sudo ./logster --dry-run --output=ganglia ApacheLogster /var/log/httpd/access_log
###
###
### Copyright 2011, Etsy, Inc.
###
### This file is part of Logster.
###
### Logster is free software: you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
### the Free Software Foundation, either version 3 of the License, or
### (at your option) any later version.
###
### Logster is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
### GNU General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with Logster. If not, see <http://www.gnu.org/licenses/>.
###
import time
import re
from logster_helper import MetricObject, LogsterParser
from logster_helper import LogsterParsingException
class ApacheLogster(LogsterParser):
def __init__(self, option_string=None):
'''Initialize any data structures or variables needed for keeping track
of the tasty bits we find in the log we are parsing.'''
self.machines = {}
# Regular expression for matching lines we are interested in, and capturing
# fields from the line (in this case, http_status_code).
self.reg = re.compile('\S{3} \d{02} \d{2}.\d{2}.\d{2} (?P<hostname>[\S]+).*HTTP/1.\d\" (?P<http_status_code>\d{3}) .*')
def parse_line(self, line):
'''This function should digest the contents of one line at a time, updating
object's state variables. Takes a single argument, the line to be parsed.'''
try:
# Apply regular expression to each line and extract interesting bits.
regMatch = self.reg.match(line)
if regMatch:
linebits = regMatch.groupdict()
status = int(linebits['http_status_code'])
machine = str(linebits['hostname'])
if machine not in self.machines.keys():
self.machines[machine] = {'1xx':0,'2xx':0,'3xx':0,'4xx':0,'5xx':0}
if (status < 200):
self.machines[machine]['1xx'] += 1
elif (status < 300):
self.machines[machine]['2xx'] += 1
elif (status < 400):
self.machines[machine]['3xx'] += 1
elif (status < 500):
self.machines[machine]['4xx'] += 1
else:
self.machines[machine]['5xx'] += 1
else:
raise LogsterParsingException, "regmatch failed to match"
except Exception, e:
raise LogsterParsingException, "regmatch or contents failed with %s on %s" % (e,line)
def get_state(self, duration):
'''Run any necessary calculations on the data collected from the logs
and return a list of metric objects.'''
self.duration = duration
return_list = []
for key in self.machines.keys():
# Return a list of metrics objects
return_list.append( MetricObject("web%s.http_1xx" % key, (self.machines[key]['1xx'] / self.duration), "Responses per sec") )
return_list.append( MetricObject("web%s.http_2xx" % key, (self.machines[key]['2xx'] / self.duration), "Responses per sec") )
return_list.append( MetricObject("web%s.http_3xx" % key, (self.machines[key]['3xx'] / self.duration), "Responses per sec") )
return_list.append( MetricObject("web%s.http_4xx" % key, (self.machines[key]['4xx'] / self.duration), "Responses per sec") )
return_list.append( MetricObject("web%s.http_5xx" % key, (self.machines[key]['5xx'] / self.duration), "Responses per sec") )
return return_list
@mrballcb

This comment has been minimized.

Copy link

@mrballcb mrballcb commented Jul 31, 2014

On CentOS 6.5, the most recent version of logster fails with:
Traceback (most recent call last):
File "/usr/bin/logster", line 443, in
main()
File "/usr/bin/logster", line 358, in main
module = import(module_name, globals(), locals(), [parser_name])
File "/usr/lib/python2.6/site-packages/logster/parsers/ApacheLogster.py", line 29, in
from logster_helper import MetricObject, LogsterParser
ImportError: No module named logster_helper

I had to change the following two lines:
from logster.logster_helper import MetricObject, LogsterParser
from logster.logster_helper import LogsterParsingException

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment